/*
 *  CodeDetector.java
 *
 *
 *  Perl Version
 *    Jerome Mathieu
 *    Peter Turney
 *
 *  Java Version
 *    Chau, Sau Ming
 *
 *  May 25, 2000
 *
 *
 *  This file defines the CodeDetector class.  CodeDetector consists of
 *  two static function detect_code(String text, String nohtml_text) and
 *  covert_html_to_nohtml(String text).
 *
 *
 *  Usage:
 *    String text        = some_input_stream();  // raw text
 *    String nohtml_text = some_input_stream();  // raw text without html
 *    int return_code    = CodeDetector.detect_code(text, nohtml_text);
 *
 *    return_code = -1  --> error text empty
 *    return_code =  0  --> Plain ASCII
 *    return_code =  1  --> ISO latin-1
 *    return_code =  2  --> Unicode (UCS-2)
 *    return_code =  3  --> Shift-JIS
 *    return_code =  4  --> JIS
 *    return_code =  5  --> EUC-jp
 *
 */

import java.util.*;
import java.io.*;
import java.net.*;
import java.sql.*;

public class CodeDetector
{
  public final static int EUCORLAT   = -6; // EUC or LAT
  public final static int SJISORLAT  = -5; // SJIS or LAT
  public final static int NOTUNI     = -4; // Not unicode
  public final static int EUCORSJIS  = -2; // EUC or SJIS
  public final static int UNIORSJIS  = -3; // Unicode or SJIS
  public final static int ASCII      = 0;  // Plain ASCII
  public final static int LAT        = 1;  // ISO latin-1
  public final static int UNI        = 2;  // Unicode (UCS-2)
  public final static int SJIS       = 3;  // Shift-JIS
  public final static int JIS        = 4;  // JIS
  public final static int NEW        = 4;  // JIS
  public final static int OLD        = 4;  // JIS
  public final static int NEC        = 4;  // JIS
  public final static int EUC        = 5;  // EUC-jp

  public final static int ESC        = 27; // Escape character (used in JIS)
  public final static int SS2        = 142;//


  /*
   *  detect_code
   *
   *  Return Character Code detected in a document in String format.
   *  Notice that this function is not for Japanese text because it
   *  will not convert the String to byte array properly.
   *
   *  Arguments
   *    1. (String) content of the text
   *    2. (String) content of the text without HTML tag
   *
   *  Return
   *    - (int) Character Code
   *
   *  detect_code(String text, String nohtml_text)
   */
  public static int detect_code(String text, String nohtml_text)
  {
    return detect_code(text.getBytes(), nohtml_text.getBytes());
  }

  
  /*
   *  detect_code
   *
   *  Return Character Code detected in a document in String format.
   *  Notice that it is the responsibility of the users the
   *  encoding the text.
   *
   *  Arguments
   *    1. (String) content of the text
   *    2. (String) content of the text without HTML tag
   *    3. (String) input String encoding method (e.g. UTF-16).  Plese refer
   *                  to Java api for more information about String encoding
   *
   *  Return
   *    - (int) Character Code
   *
   *  detect_code(String text, String nohtml_text)
   */
  public static int detect_code(String text, String nohtml_text, String enc) throws UnsupportedEncodingException
  {
    try
    {
      return detect_code(text.getBytes(enc), nohtml_text.getBytes(enc));
    }
    catch (UnsupportedEncodingException e)
    {
      throw e;
    }
  }

  /*
   *  byte_array_to_short_array
   *
   *  Return short array (short[]) converted from byte array (byte[]) passed
   *  as an argument.
   *
   *  Arguments
   *    1. (byte[]) content of the text
   *    2. (byte[]) content of the text without HTML tag
   *
   *  Return
   *    - (int) Character Code
   *
   *  byte_array_to_short_array(byte[] buf)
   */
  public static short[] byte_array_to_short_array(byte[] buf)
  {
    int size = buf.length;
    short[] arr_return = new short[size];
    short sht_temp;
    for (int i = 0; i < size; i++)
    {
      if (buf[i] < 0)
      {
        arr_return[i] = (short)(buf[i] + 256);
      }
      else
      {
        arr_return[i] = (short)buf[i];
      }
    }
    return arr_return;
  }
  
  
  /*
   *  detect_code
   *
   *  Return Character Code detected in a document in byte array format.
   *
   *  Arguments
   *    1. (byte[]) content of the text
   *    2. (byte[]) content of the text without HTML tag
   *
   *  Return
   *    - (int) Character Code
   *
   *  detect_code(String text, String nohtml_text)
   */
  public static int detect_code(byte[] buf_text, byte[] buf_nohtml_text)
  {
    int whatcode;      // the encoding
    short[] buf = byte_array_to_short_array(buf_text);

    if (buf.length <= 0)
    { // Return -1 if no text
      return -1;
    }

    // check for unicode UCS-2
    whatcode = check_unicode(buf);

    // check for JIS, ASCII, or possible SJIS, EUC (or ISO latin-1)
    whatcode = check_JIS_ASCII_or_SJIS_EUC(buf, whatcode);
    //System.out.println("whatcode = check_JIS_ASCII_or_SJIS_EUC(buf, buf_size, whatcode); " + whatcode);

    buf = byte_array_to_short_array(buf_nohtml_text);

    // It might look like SJIS, but might be in ISO latin-1
    whatcode = check_SJIS_or_ISO_latin_1(buf, whatcode);
    //System.out.println("whatcode = check_SJIS_or_ISO_latin_1(buf, buf_size, whatcode); " + whatcode);

    // It might look like EUC, but might be really in ISO latin-1
    whatcode = check_EUC_or_ISO_latin_1(buf, whatcode);
    //System.out.println("whatcode = check_EUC_or_ISO_latin_1(buf, buf_size, whatcode); " + whatcode);

    return whatcode;
  } // public static int detect_code(String text, String nohtml_text)


  /*
   *  check_unicode
   *
   *  Method for detect_code() to check if the text is in Unicode.
   *
   *  Arguments
   *    1. (short[]) content of the text in short array converted from byte array
   *
   *  Return
   *    - (int) Character Code
   *
   *  check_unicode(short[] buf, int buf_size)
   */
  protected static int check_unicode(short[] buf)
  {
    int whatcode = ASCII;      // the encoding
    int pos_EOF = buf.length;       // length of the text strings (length of the list of integer values
    int i = 0;             // the position in the list of integer values
    short c = 0;             // the value of the current integer value

    while ((whatcode == ASCII) && (i != pos_EOF))
    {
      c = buf[i];
      i++;
      if (c == 60)
      {  // '<'
        c = buf[i];
        i++;
        if (c == 0)
        {  // only unicode UCS-2 has a 0 after '<'
          whatcode = UNI;
        }
        else
        {
          whatcode = NOTUNI;
        }
      }
    } // while ((whatcode == ASCII) && (i != pos_EOF))

    if (whatcode == UNI)
    {
      return UNI;
    }
    else
    {
      whatcode = ASCII;
    }
    return whatcode;
  } //protected static int check_unicode(byte[] buf, int buf_size)


  /*
   *  check_JIS_ASCII_or_SJIS_EUC
   *
   *  Method for detect_code() to check if the text is in check for JIS,
   *  ASCII, or possible SJIS, EUC (or ISO latin-1).
   *
   *  Arguments
   *    1. (short[]) content of the text in short array converted from byte array
   *    2. (int) the intermidate code generated by last function
   *
   *  Return
   *    - (int) Character Code
   *
   *  check_JIS_ASCII_or_SJIS_EUC(short[] buf, int whatcode_passed)
   */
  protected static int check_JIS_ASCII_or_SJIS_EUC(short[] buf, int whatcode_passed)
  {
    int whatcode = whatcode_passed;
    int pos_EOF = buf.length;   // length of the text strings (length of the list of integer values
    int i = 0;                // the position in the list of integer values
    short c = 0;              // the value of the current integer value

    while (((whatcode == EUCORSJIS) || (whatcode == ASCII)) && (i != pos_EOF))
    {
      c = buf[i];
      i++;
      if (i != pos_EOF)
      {
        if (c == ESC)
        {
          c = buf[i];
          i++;
          if (c == 36)
          { // ''
            c = buf[i];
            i++;
            if (c == 66)
            {  // 'B'
              whatcode = NEW;
            }
            else if (c == 64)
            { // '@'
              whatcode = OLD;
            }
          }
          else if (c == 73)
          { // 'K'
            whatcode = NEC;
          }
        }
        else if (((c >= 129) && (c <= 141)) || ((c >= 143) && (c <= 159)))
        {
          //System.out.println("whatcode = SJISORLAT");
          whatcode = SJISORLAT;
        }
        else if (c == SS2)
        {
          c = buf[i];
          i++;
          if (((c >= 64) && (c <= 126)) || ((c >= 128) && (c <= 160)) || ((c >= 224) && (c <= 252)))
          {
            //System.out.println("whatcode = SJISORLAT 2");
            whatcode = SJISORLAT;
          }
          else if ((c >= 161) && (c <= 223))
          {
            //System.out.println("whatcode = EUCORSJIS");
            whatcode = EUCORSJIS;
          }
        }
        else if ((c >= 161) && (c <= 223))
        {
          c = buf[i];
          i++;
          if ((c >= 240) && (c <= 254))
          {
            //System.out.println("whatcode = EUCORLAT");
            whatcode = EUCORLAT;
          }
          else if ((c >= 161) && (c <= 223))
          {
            //System.out.println("whatcode = EUCORSJIS 2");
            whatcode = EUCORSJIS;
          }
          else if ((c >= 224) && (c <= 239))
          {
            //System.out.println("whatcode = EUCORSJIS 3");
            whatcode = EUCORSJIS;
            while ((c >= 64) && (i != pos_EOF) && (whatcode == EUCORSJIS))
            {
              if (c >= 129)
              {
                if ((c <= 141) || ((c >= 143) && (c <= 159)))
                {
                  //System.out.println("whatcode = SJISORLAT 3");
                  whatcode = SJISORLAT;
                }
                else if ((c >= 253) && (c <= 254))
                {
                  //System.out.println("whatcode = EUCORLAT 2");
                  whatcode = EUCORLAT;
                }
              }
              c = buf[i];
              i++;
            }
          }
          else if (c <= 159)
          {
            //System.out.println("whatcode = SJISORLAT 4");
            whatcode = SJISORLAT;
          }
        }
        else if ((c >= 240) && (c <= 254))
        {
          //System.out.println("whatcode = EUCORLAT 3");
          whatcode = EUCORLAT;
        }
        else if ((c >= 224) && (c <= 239))
        {
          c = buf[i];
          i++;
          if (((c >= 64) && (c <= 126)) || ((c >= 128) && (c <= 160)))
          {
            //System.out.println("whatcode = SJISORLAT 5");
            whatcode = SJISORLAT;
          }
          else if ((c >= 253) && (c <= 254))
          {
            //System.out.println("whatcode = EUCORLAT 4");
            whatcode = EUCORLAT;
          }
          else if ((c >= 161) && (c <= 252))
          {
            //System.out.println("whatcode = EUCORSJIS 4");
            whatcode = EUCORSJIS;
          }
        }
      }
    }
    return whatcode;
  } // protected static int check_JIS_ASCII_or_SJIS_EUC(byte[] buf, int buf_size, int whatcode)


  /*
   *  check_SJIS_or_ISO_latin_1
   *
   *  It might look like SJIS, but might be in ISO latin-1
   *
   *  Heuristics:
   *    -We count the number of valid SJIS double byte characters.
   *    -We count the number of roman letter
   *    -If we have more roman letter than SJIS double byte characters,
   *     we say that the encoding is ISO latin-1
   *
   *  Arguments
   *    1. (short[]) content of the text in short array converted from byte array
   *    2. (int) the intermidate code generated by last function
   *
   *  Return
   *    - (int) Character Code
   *
   *  check_SJIS_or_ISO_latin_1(short[] buf, int whatcode_passed)
   */
  protected static int check_SJIS_or_ISO_latin_1(short[] buf, int whatcode_passed)
  {
    int whatcode = whatcode_passed;
    int pos_EOF = buf.length;       // length of the text strings (length of the list of integer values
    int sjis_count = 0;    // number of valid sjis double byte characters
    int single_count = 0;  // number of single byte roman characters
    int i = 0;             // the position in the list of integer values
    short c = 0;             // the value of the current integer value

    if (whatcode == SJISORLAT)
    { //  check if text(without html tags) is in SJIS or ISO latin-1
      i = 0;
      while ((whatcode == SJISORLAT) && (i != pos_EOF))
      {
        c = buf[i];
        i++;
        if (i != pos_EOF)
        {
          if (((c >= 129) && (c <=159)) || ((c >= 224) && (c <= 239)))
          {
            c = buf[i];
            i++;
            if (((c >= 64) && (c <=126)) || ((c >=128) && (c <= 252)))
            { // Valid double byte SJIS character
              sjis_count += 1;
            }
            else
            { // Invalid double byte SJIS character, so it must be LAT.
              //i--; i--; c = buf[i];
              //print "<p>ISO Latin-1 because: (c,";
              //i++; c = buf[i];
              //print "c) invalid double byte SJIS character.</p>\n";
              whatcode = LAT;
            }
          }
          else if ((c >= 65) && (c <= 126))
          { // 65-126 : alphabetical letters
            single_count += 1;
          }
          else if (((c >= 33) && (c <= 64)) || ((c >= 161) && (c <= 223)))
          { // 161-223: Half-width katakana in SJIS and special char in LAT.
            //          (cannot be differenciated.)
          }
          //else if (c == 160) {
          // 160    : This character is rarely used in LAT.(&nbsp is used instead)
          //  whatcode = SJIS; }
          else if ((c >= 240) && (c <= 252))
          {
            c = buf[i];
            i++;
            if (((c >= 64) && (c <=126)) || ((c >=128) && (c <= 252)))
            { // Valid double byte SJIS character
              sjis_count += 1;
            }
            else
            {
              // Invalid double byte SJIS character, so must be LAT.
              //i--; i--; c = buf[i];
              //print "<p>ISO Latin-1 because: (c,";
              //i++; c = buf[i];
              //print "c) invalid double byte SJIS character.</p>\n";
              whatcode = LAT;
            }
          }
          else
          {
            // other single byte characters
          }
        }
      }
      if (whatcode == SJISORLAT)
      {
        if (sjis_count > single_count)
        { //print "<p>SJIS, because: // SJIS double byte (sjis_count) > (single_count) // roman letters</p>\n";
          whatcode = SJIS;
        }
        else
        { //print "<p>Latin-1, because: // SJIS double byte (sjis_count) <= (single_count) // roman letters</p>\n";
          whatcode = LAT;
        }
      }
    }
    return whatcode;
  } // protected static int check_SJIS_or_ISO_latin_1(byte[] buf, int buf_size, int whatcode)


  /*
   *  check_EUC_or_ISO_latin_1
   *
   *  It might look like EUC, but might be really in ISO latin-1
   *
   *  Heuristics:
   *    -We count the number of valid EUC double byte characters.
   *    -We count the number of roman letter
   *    -If we have more roman letter than EUC double byte characters,
   *     we say that the encoding is ISO latin-1
   *
   *  Arguments
   *    1. (short[]) content of the text in short array converted from byte array
   *    2. (int) the intermidate code generated by last function
   *
   *  Return
   *    - (int) Character Code
   *
   *  check_EUC_or_ISO_latin_1(short[] buf, int whatcode_passed)
   */
  protected static int check_EUC_or_ISO_latin_1(short[] buf, int whatcode_passed)
  {
    int whatcode = whatcode_passed;
    int pos_EOF = buf.length;       // length of the text strings (length of the list of integer values
    int euc_count = 0;     // number of valid euc double byte characters
    int single_count = 0;  // number of single byte roman characters
    int i = 0;             // the position in the list of integer values
    short c = 0;             // the value of the current integer value

    if (whatcode == EUCORLAT)
    { // check if text(without html tags) is in SJIS or ISO latin-1
      i = 0;
      while ((whatcode == EUCORLAT) && (i != pos_EOF))
      {
        c = buf[i];
        i++;
        if (i != pos_EOF)
        {
          if ((c >= 161) && (c <=254))
          {
            c = buf[i];
            i++;
            if ((c >= 161) && (c <=254))
            { // Valid double byte EUC character
              euc_count += 1;
            }
            else
            { // Invalid double byte SJIS character, so it must be LAT.
              //i--; i--; c = buf[i];
              //print "<p>ISO Latin-1 because: (c,";
              //i++; c = buf[i];
              //print "c) invalid double byte SJIS character.</p>\n";
              whatcode = LAT;
            }
          }
          else if (c == 142)
          {
            c = buf[i];
            i++;
            if ((c >= 161) && (c <=223))
            { // Valid double byte EUC character (half width katakana)
              euc_count += 1;
            }
            else
            { // Invalid double byte EUC character, so it must be LAT.
              whatcode = LAT;
            }
          }
          else if ((c >= 65) && (c <= 126))
          { // 65-126 : alphabetical letters
            single_count += 1;
          }
          else if ((c >= 33) && (c <= 64))
          { // 33-64: special char in LAT.
            //          (cannot be differenciated.)
          }
          else if ((c >= 127) && (c <= 141))
          { // Invalid EUC character
            whatcode = LAT;
          }
          else
          {
            // other single byte characters
          }
        }
      }
      if (whatcode == EUCORLAT)
      {
        if (euc_count > single_count)
        {
          //print "<p>EUC, because: // EUC double byte (euc_count) > (single_count) // roman letters</p>\n";
          whatcode = EUC;
        }
        else
        {
          //print "<p>Latin-1, because: // EUC double byte (euc_count) <= (single_count) // roman letters</p>\n";
          whatcode = LAT;
        }
      }
    }
    return whatcode;
  } // protected static int check_EUC_or_ISO_latin_1(byte[] buf, int buf_size, int whatcode)
  
  
  /*
   *  convert_html_to_nohtml
   *
   *  Filter a text without HTML tags and everything between <script> and </script>
   *
   *  Arguments
   *    1. (byte[]) content of the text in byte array
   *
   *  Return
   *    - (byte[]) text without HTML tag and everything between <script>
   *                and </script>
   *
   *  convert_html_to_nohtml(byte[] text)
   */
  public static byte[] convert_html_to_nohtml(byte[] text)
  {
    byte in;
    int counter = 0;
    
    byte[] barr_return;
    
    // Get rid of everything between <script> and </script>.
    int int_script_pos = indexOf(text, (new String("<script>")).getBytes());
    int int_script_end_pos = indexOf(text, (new String("</script>")).getBytes());    
    if ((int_script_pos != -1) && (int_script_end_pos != -1))
    { // Script content is found.
      barr_return = new byte[text.length - (int_script_end_pos - int_script_pos + 8)];
      System.arraycopy(text, 0, barr_return, 0, int_script_pos);
      System.arraycopy(text, int_script_end_pos + 9, barr_return,
        int_script_pos, text.length - int_script_end_pos - 9);
    }
    else
    {
      barr_return = new byte[text.length];
      System.arraycopy(text, 0, barr_return, 0, text.length);
    }
    
    int size = barr_return.length;
    boolean bln_html = false;
    for (int i = 0; i < size; i++)
    { // Continue get character until end of byte array.  Also, discard
      // everything between '<' and '>' inclusively.
      in = barr_return[i];
      if (((char)in == '<') && (!bln_html))
      {
        bln_html = true;
      }
      else if (((char)in == '>') && (bln_html))
      {
        bln_html = false;
      }
      if ((!bln_html) && ((char)in != '>'))
      {
        barr_return[counter] = in;
        counter++;
      }
    }

    // Truncate the barr_return to size of counter.
    byte[] barr_temp = barr_return;
    barr_return = new byte[counter];
    System.arraycopy(barr_temp, 0, barr_return, 0, counter);
    return barr_return;
  }
  
  
  /*
   *  indexOf
   *
   *  Return the index of byte_array at that the first occurence of subarray
   *  is found if there is any; otherwise return -1.  Notice that the
   *  function makes no case conversion of the array passed.  Hence, this
   *  function is case sensitive if the array represents a character array.
   *  Conversion is only done on the subarray to lower and upper cases.
   *
   *  Arguments
   *    1. (byte[]) byte array to be checked
   *    2. (byte[]) byte subarray
   *
   *  Return
   *    - (int) index of the array that contains the subarray.
   *
   *  indexOf(byte[] byte_array, byte[] byte_subarray)
   */
  public static int indexOf(byte[] byte_array, byte[] byte_subarray)
  {
    int int_return = -1;
    int byte_array_length = byte_array.length;
    int byte_subarray_length = byte_subarray.length;
    int int_counter = 0;
    byte[] barr_temp = new byte[byte_subarray_length];
    byte[] byte_subarray_lower = (new String(byte_subarray)).toLowerCase().getBytes();
    byte[] byte_subarray_upper = (new String(byte_subarray)).toUpperCase().getBytes();
    while ((int_return < 0) && (byte_subarray_length + int_counter + 1 <= byte_array_length))
    { // Repeat until either the first occurence of subarray is found or
      // the length of remaining unchecked array is less than the length
      // of subarray.
      System.arraycopy(byte_array, int_counter, barr_temp, 0, byte_subarray_length);
      if ((Arrays.equals(barr_temp, byte_subarray))
            || (Arrays.equals(barr_temp, byte_subarray_lower))
            || (Arrays.equals(barr_temp, byte_subarray_upper)))
      {
        int_return = int_counter;
      }
      int_counter++;
    }
    return int_return;
  }

} // public class CodeDetector